{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#数据量太大，pdandas不能一次讲所有数据读入\n",
    "#也可以用pandas,一次读取部分数据，可以参考：https://www.cnblogs.com/datablog/p/6127000.html\n",
    "#import pandas as pd\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy.sparse as ss\n",
    "import scipy.io as sio\n",
    "\n",
    "#保存数据\n",
    "import pickle\n",
    "\n",
    "#event的特征需要编码\n",
    "\n",
    "from sklearn.preprocessing import normalize\n",
    "#相似度/距离\n",
    "import scipy.spatial.distance as ssd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "活动记录：number of records :3137972    （event）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 特征编码工具\n",
    "#该事件涉及国家、城市、时间等信息的处理\n",
    "\n",
    "#保存数据\n",
    "import pickle\n",
    "\n",
    "#特征编码\n",
    "import datetime\n",
    "import hashlib \n",
    "# POSIX locale database and functionality\n",
    "import locale\n",
    "#国家的编码、名字、语言、货币等信息\n",
    "import pycountry\n",
    "\n",
    "import numpy as np\n",
    "import scipy.io as sio\n",
    "import scipy.sparse as ss\n",
    "\n",
    "from collections import defaultdict\n",
    "from sklearn.preprocessing import normalize\n",
    "\n",
    "#类别型特征编码\n",
    "#这里写成类的形式，因为编码字典要在不同的文件中使用\n",
    "class FeatureEng:\n",
    "  def __init__(self):\n",
    "    \n",
    "    # 载入 locales\n",
    "    self.localeIdMap = defaultdict(int)\n",
    "    for i, l in enumerate(locale.locale_alias.keys()):\n",
    "      self.localeIdMap[l] = i + 1\n",
    "    #print locale.locale_alias.keys()\n",
    "\n",
    "    # 载入 countries\n",
    "    self.countryIdMap = defaultdict(int)\n",
    "    ctryIdx = defaultdict(int)\n",
    "    for i, c in enumerate(pycountry.countries):\n",
    "      self.countryIdMap[c.name.lower()] = i + 1\n",
    "      if c.name.lower() == \"usa\":\n",
    "        ctryIdx[\"US\"] = i\n",
    "      if c.name.lower() == \"canada\":\n",
    "        ctryIdx[\"CA\"] = i\n",
    "    for cc in ctryIdx.keys():\n",
    "      for s in pycountry.subdivisions.get(country_code=cc):\n",
    "        self.countryIdMap[s.name.lower()] = ctryIdx[cc] + 1\n",
    "        \n",
    "    # 载入 gender id 字典\n",
    "    ##缺失补0，性别未知\n",
    "    self.genderIdMap = defaultdict(int, {'NaN': 0, \"male\":1, \"female\":2})\n",
    "\n",
    "  def getLocaleId(self, locstr):\n",
    "    return self.localeIdMap[locstr.lower()]\n",
    "\n",
    "  def getGenderId(self, genderStr):\n",
    "    return self.genderIdMap[genderStr]\n",
    "\n",
    "  def getJoinedYearMonth(self, dateString):\n",
    "    try:\n",
    "        dttm = datetime.datetime.strptime(dateString, \"%Y-%m-%dT%H:%M:%S.%fZ\")\n",
    "        return (dttm.year-2010)*12 + dttm.month\n",
    "    except:  #缺失补0\n",
    "        return 0\n",
    "\n",
    "  def getCountryId(self, location):\n",
    "    if (isinstance(location, str)\n",
    "        and len(location.strip()) > 0\n",
    "        and location.rfind(\"  \") > -1):\n",
    "        return self.countryIdMap[location[location.rindex(\"  \") + 2:].lower()]\n",
    "    else:\n",
    "        return 0\n",
    "\n",
    "  def getBirthYearInt(self, birthYear):\n",
    "    try:\n",
    "      return 0 if birthYear == \"None\" else int(birthYear)\n",
    "    except:\n",
    "      return 0\n",
    "\n",
    "  def getTimezoneInt(self, timezone):\n",
    "    try:\n",
    "      return int(timezone)\n",
    "    except:\n",
    "      return 0\n",
    "\n",
    "  def getFeatureHash(self, value):\n",
    "    if len(value.strip()) == 0:\n",
    "      return -1\n",
    "    else:\n",
    "      #return int(hashlib.sha224(value).hexdigest()[0:4], 16)错的，最原先的\n",
    "      \n",
    "      #return int(hashlib.sha224(value.encode(\"utf-8\")).hexdigest()[0:4], 16)错的\n",
    "      #return int(hashlib.sha224(str(value).encode(\"utf-8\")).hexdigest()[0:4], 16)错的\n",
    "      return int(hashlib.md5(value.encode(\"utf-8\")).hexdigest()[0:4], 16)#经本人修改方可不出现“不可hash编码的情况”\n",
    "#       m = hashlib.md5()\n",
    "#       m.update(value)\n",
    "#       return int(m.hexdigest())错的\n",
    "      \n",
    "  def getFloatValue(self, value):\n",
    "    if len(value.strip()) == 0:\n",
    "      return 0.0\n",
    "    else:\n",
    "      return float(value)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "因为自身不能把原先utils.py调用，所以将源代码拿过来运行"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of events in train & test :13418\n"
     ]
    }
   ],
   "source": [
    "eventIndex = pickle.load(open(\"PE_eventIndex.pkl\", 'rb'))\n",
    "eventNum = len(eventIndex)\n",
    "\n",
    "print(\"number of events in train & test :%d\" % eventNum)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "特征编码,活动之间的相似度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "FE = FeatureEng()#实例\n",
    "fin = open(\"events.csv\",\"r\")\n",
    "fin.readline()#跳过第一行\n",
    "#前7个特征的关系矩阵\n",
    "eventPreMatrix = ss.dok_matrix((eventNum,7))\n",
    "#词频特征\n",
    "eventContMatrix = ss.dok_matrix((eventNum,101))\n",
    "\n",
    "for line in fin.readlines():\n",
    "    cols = line.strip().split(\",\")\n",
    "    eventId = str(cols[0])\n",
    "    if eventId in eventIndex:\n",
    "        i = eventIndex[eventId]\n",
    "        \n",
    "        #成员函数见上一个cell(utils),依次填充列表\n",
    "        eventPreMatrix[i, 0] = FE.getJoinedYearMonth(cols[2]) # start_time    \n",
    "        eventPreMatrix[i, 1] = FE.getFeatureHash(cols[3]) # city  \n",
    "        eventPreMatrix[i, 2] = FE.getFeatureHash(cols[4]) # state\n",
    "        eventPreMatrix[i, 3] = FE.getFeatureHash(cols[5]) # zip\n",
    "        eventPreMatrix[i, 4] = FE.getFeatureHash(cols[6]) # country\n",
    "        eventPreMatrix[i, 5] = FE.getFloatValue(cols[7]) # lat\n",
    "        eventPreMatrix[i, 6] = FE.getFloatValue(cols[8]) # lon\n",
    "        \n",
    "        #词频\n",
    "        for j in range(9,110):\n",
    "            eventContMatrix[i,j-9] = cols[j]\n",
    "fin.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#用L2模归一化\n",
    "eventPropMatrix = normalize(eventPreMatrix,norm=\"l2\", axis=0, copy=False)\n",
    "sio.mmwrite(\"EV_eventPropMatrix\", eventPropMatrix)\n",
    "#词频，我们用此部分特征进行聚类，得到活动的genre\n",
    "eventContMatrix = normalize(eventContMatrix,norm=\"l2\", axis=0, copy=False)\n",
    "sio.mmwrite(\"EV_eventContMatrix\", eventContMatrix)\n",
    "\n",
    "#根据两个矩阵计算事件对之间的相似性\n",
    "eventPropSim = ss.dok_matrix((eventNum, eventNum))\n",
    "eventContSim = ss.dok_matrix((eventNum, eventNum))\n",
    "#读取在train集和test集中出现的活动对\n",
    "uniqueEventPairs = pickle.load(open(\"PE_uniqueEventPairs.pkl\", 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "for p1,p2 in uniqueEventPairs:\n",
    "    i = p1\n",
    "    j = p2\n",
    "   \n",
    "    if (i not in eventPropSim)and (j not in eventPropSim):\n",
    "    #if not eventPropSim.has_key((i,j)):\n",
    "    #非词频特征，采用Person相关系数\n",
    "        epsim = ssd.correlation(eventPropMatrix.getrow(i).todense(),eventPropMatrix.getrow(j).todense())\n",
    "        \n",
    "        eventPropSim[i, j] = epsim\n",
    "        eventPropSim[j, i] = epsim\n",
    "    \n",
    "    #对词频特征，采用余弦相似度\n",
    "    if (i not in eventContSim)and (j not in eventContSim):\n",
    "        ecsim = ssd.cosine(eventContMatrix.getrow(i).todense(),eventContMatrix.getrow(j).todense())\n",
    "    \n",
    "        eventContSim[i, j] = epsim\n",
    "        eventContSim[j, i] = epsim\n",
    "sio.mmwrite(\"EV_eventPropSim\", eventPropSim)\n",
    "sio.mmwrite(\"EV_eventContSim\", eventContSim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matrix([[0., 0., 0., ..., 0., 0., 0.]])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eventPropSim.getrow(0).todense()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "对“物品”（event）的数据处理完毕"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The numbers of users is 3391\n"
     ]
    }
   ],
   "source": [
    "#读取已处理的train和test集上的用户\n",
    "userIndex = pickle.load(open(\"PE_userIndex.pkl\",\"rb\"))\n",
    "uesrNum = len(userIndex)\n",
    "print(\"The numbers of users is %d\"% uesrNum )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "特征编码，用户之间的相似度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>locale</th>\n",
       "      <th>birthyear</th>\n",
       "      <th>gender</th>\n",
       "      <th>joinedAt</th>\n",
       "      <th>location</th>\n",
       "      <th>timezone</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [user_id, locale, birthyear, gender, joinedAt, location, timezone]\n",
       "Index: []"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user = pd.read_csv(\"users.csv\")\n",
    "user.head(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "FE = FeatureEng()\n",
    "#对特征进行编码，所以去掉ID号\n",
    "colsUsers = user.shape[1] - 1\n",
    "colsUsers = [\"locale\",\"birthyear\",\"gender\",\"joinedAt\",\"location\",\"tiemzone\"]\n",
    "\n",
    "#建关系矩阵容器\n",
    "userMatrix = ss.dok_matrix((uesrNum,uesrNum))\n",
    "\n",
    "for u in range(user.shape[0]):\n",
    "    userId = str(user.loc[u,\"user_id\"])\n",
    "    if userId in userIndex:\n",
    "        i = userIndex[userId]\n",
    "        \n",
    "        userMatrix[i,0] = FE.getLocaleId(user.loc[u,'locale'])\n",
    "        userMatrix[i, 1] = FE.getBirthYearInt(user.loc[u,'birthyear'])\n",
    "        userMatrix[i, 2] = FE.getGenderId(user.loc[u,'gender'])\n",
    "        userMatrix[i, 3] = FE.getJoinedYearMonth(user.loc[u,'joinedAt'])\n",
    "        userMatrix[i, 4] = FE.getCountryId(user.loc[u,'location'])#失效\n",
    "        userMatrix[i, 5] = FE.getTimezoneInt(user.loc[u,'timezone'])\n",
    "userMatrix = normalize(userMatrix, norm=\"l2\", axis=0, copy=False)#归一化用户关系矩阵\n",
    "sio.mmwrite(\"US_userMatrix\", userMatrix)\n",
    "\n",
    "# 计算用户相似度矩阵，用于推荐系统\n",
    "userSimMatrix = ss.dok_matrix((uesrNum, uesrNum))\n",
    "\n",
    "#读取在测试集和训练集中出现的用户对\n",
    "uniqueUserPairs = pickle.load(open(\"PE_uniqueUserPairs.pkl\", 'rb'))\n",
    "\n",
    "#矩阵对角线元素\n",
    "for i in range(0, uesrNum):\n",
    "    userSimMatrix[i, i] = 1.0#设置为1.0\n",
    "\n",
    "#对称\n",
    "for user1, user2 in uniqueUserPairs:\n",
    "    i = user1\n",
    "    j = user2\n",
    "    if (i not in userSimMatrix) and (j not in userSimMatrix):\n",
    "        #Person相关系数做为相似度度量\n",
    "    \n",
    "        usim = ssd.correlation(userMatrix.getrow(i).todense(),userMatrix.getrow(j).todense())\n",
    "        userSimMatrix[i, j] = usim\n",
    "        userSimMatrix[j, i] = usim\n",
    "    \n",
    "sio.mmwrite(\"US_userSimMatrix\", userSimMatrix)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "users数据处理完毕"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of users  is 3391 ,in train & test \n"
     ]
    }
   ],
   "source": [
    "#读取训练集和测试集中出现过的事件列表,再次读取，方便看\n",
    "userIndex = pickle.load(open(\"PE_userIndex.pkl\", 'rb'))\n",
    "usersNum =  len(userIndex)\n",
    "\n",
    "print(\"number of users  is %d ,in train & test \" % usersNum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "#读取用户活动关系矩阵\n",
    "userEventScores = sio.mmread(\"PE_userEventScores\")\n",
    "#读取，之后用于用户朋友参加的活动影响到用户\n",
    "eventForUser = pickle.load(open(\"PE_eventsForUser.pkl\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>friends</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [user, friends]\n",
       "Index: []"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_friends = pd.read_csv(\"user_friends.csv\")\n",
    "user_friends.head(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "friendsNum = np.zeros(usersNum)\n",
    "usersFriends = ss.dok_matrix((usersNum,usersNum))\n",
    "\n",
    "file = open(\"user_friends.csv\",\"r\")\n",
    "file.readline()#跳过第一行\n",
    "\n",
    "for line in file:#对每个样本\n",
    "    cols = line.strip().split(\",\")\n",
    "    user = str(cols[0])\n",
    "    if user in userIndex:\n",
    "        friends = cols[1].split(\" \")#用户的朋友\n",
    "        i = userIndex[user] #用户的索引\n",
    "        friendsNum[i] = len(friends)\n",
    "        for friend in friends:#对每个朋友\n",
    "            str_friend = str(friend)\n",
    "            if str_friend in userIndex:#如果朋友也在用户列表中\n",
    "                j = userIndex[str_friend]\n",
    "            \n",
    "file.close()\n",
    "\n",
    "friendsNumSum = friendsNum.sum(axis = 0)#用户的总朋友数目\n",
    "friendsNum = friendsNum/friendsNumSum#归一化数组\n",
    "sio.mmwrite(\"UF_numFriends\",np.matrix(friendsNum))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[3.97708406e-04 1.43110707e-04 3.83236537e-05 ... 7.98632784e-05\n",
      "  9.40671500e-05 1.24565274e-03]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([3.97708406e-04, 1.43110707e-04, 3.83236537e-05, ...,\n",
       "       7.98632784e-05, 9.40671500e-05, 1.24565274e-03])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(np.matrix(friendsNum))\n",
    "friendsNum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "usersFriends = normalize(usersFriends,norm = \"l2\",axis = 0,copy = False)#l2归一化\n",
    "sio.mmwrite(\"UF_userFriends\",usersFriends)#保存数据"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "users_friend数据处理完毕"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of events is :13418\n"
     ]
    }
   ],
   "source": [
    "eventIndex = pickle.load(open(\"PE_eventIndex.pkl\", 'rb'))\n",
    "eventsNum = len(eventIndex)\n",
    "\n",
    "print(\"number of events is :%d\" % eventsNum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "#统计单一活动，参加：no  or yes的人数，计算活动热度\n",
    "eventPopu = ss.dok_matrix((eventsNum,1))\n",
    "\n",
    "f = open(\"event_attendees.csv\",\"r\")\n",
    "f.readline()\n",
    "\n",
    "for line in f:\n",
    "    cols = line.strip().split(\",\")\n",
    "    str_cols = str(cols[0])\n",
    "    if str_cols in eventIndex:\n",
    "        i = eventIndex[str_cols]#活动索引\n",
    "        eventPopu[i,0] = len(cols[1].split(\" \")) - len(cols[4].split(\" \"))\n",
    "\n",
    "f.close()\n",
    "eventPopu = normalize(eventPopu,norm = \"l1\",axis = 0,copy = False)\n",
    "sio.mmwrite(\"EP_eventPopu\",eventPopu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<13418x1 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 12635 stored elements in Compressed Sparse Column format>"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#np.matrix(eventPopu)\n",
    "eventPopu"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "活动参与数据已处理"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
